Fix non-reproducible TrueType hinting: invalid qsort comparators

Several comparison functions passed to qsort() violated the ordering
axioms. With glibc >= 2.37 qsort() takes an in-place introsort path that
has undefined behaviour for such comparators and reads uninitialised
memory, so the generated TrueType instructions varied between builds
(e.g. FreeSerif.ttf from the GNU FreeFont sources).

Make SortBlues(), SortStems(), sortbynum(), sortedges(), stem_cmp() and
line_pt_cmp() return a proper trichotomous result and compare both
arguments symmetrically.

Gbp-Pq: Name 0004-qsort-comparators.patch

diff --git a/fontforge/nowakowskittfinstr.c b/fontforge/nowakowskittfinstr.c
index fce8dc535a5bd975f804b103f1ec67724a6b0942..c6592eba7cefdaa61bd5b9c60f53e16c0ff70fe4 100644 (file)
--- a/fontforge/nowakowskittfinstr.c
+++ b/fontforge/nowakowskittfinstr.c
@@ -467,7 +467,8 @@ return !((b2 > o1) || (o2 < b1));
 /* To be used with qsort() - sorts BlueZone array by base in ascending order.
  */
 static int SortBlues(const void *a, const void *b) {
-    return ((BlueZone *)a)->base > ((BlueZone *)b)->base;
+    real ba = ((BlueZone *)a)->base, bb = ((BlueZone *)b)->base;
+    return (ba > bb) - (ba < bb);
 }
 
 /* Import blue data into global instructing context. Include family blues too.
@@ -617,7 +618,8 @@ static void GICImportBlues(GlobalInstrCt *gic) {
 /* To be used with qsort() - sorts StdStem array by width in ascending order.
  */
 static int SortStems(const void *a, const void *b) {
-    return ((StdStem *)a)->width > ((StdStem *)b)->width;
+    real wa = ((StdStem *)a)->width, wb = ((StdStem *)b)->width;
+    return (wa > wb) - (wa < wb);
 }
 
 /* Import stem data into global instructing context. We deal only with
@@ -2596,7 +2598,8 @@ static void init_edge(InstrCt *ct, real base, int contour_direction) {
 
 /* To be used with qsort() - sorts integer array in ascending order. */
 static int sortbynum(const void *a, const void *b) {
-    return *(int *)a > *(int *)b;
+    int ia = *(const int *)a, ib = *(const int *)b;
+    return (ia > ib) - (ia < ib);
 }
 
 /* Find element's index within an array - return -1 if element not found. */
@@ -4916,7 +4919,7 @@ struct stemedge {
 /* To be used with qsort() - sorts edge array in ascending order. */
 static int sortedges(const void *_e1, const void *_e2) {
     const struct stemedge *e1 = _e1, *e2 = _e2;
-    return ( e1->pos > e2->pos );
+    return ( e1->pos > e2->pos ) - ( e1->pos < e2->pos );
 }
 
 static int AddEdge(InstrCt *ct, StemData *stem, int is_l, struct stemedge *edgelist, int cnt) {

diff --git a/fontforge/stemdb.c b/fontforge/stemdb.c
index 0dc17502722fad4e498e7db24dcccb5347cf2194..f412e8f73300b3cb0321cb1f11d8c69e318bcb77 100644 (file)
--- a/fontforge/stemdb.c
+++ b/fontforge/stemdb.c
@@ -447,12 +447,22 @@ static int line_pt_cmp( const void *_p1, const void *_p2 ) {
     struct linedata *line;
     double ppos1=0,ppos2=0;
     
+    /* Pick a line shared by both points, considering both arguments so
+     * that the comparison is symmetric: with an asymmetric comparator
+     * qsort() has undefined behaviour and the resulting ordering depends
+     * on uninitialised memory (non-reproducible output). */
     if ( (*pd1)->prevline != NULL && 
        ( (*pd1)->prevline == (*pd2)->prevline || (*pd1)->prevline == (*pd2)->nextline ))
        line = (*pd1)->prevline;
     else if ( (*pd1)->nextline != NULL && 
        ( (*pd1)->nextline == (*pd2)->prevline || (*pd1)->nextline == (*pd2)->nextline ))
        line = (*pd1)->nextline;
+    else if ( (*pd2)->prevline != NULL && 
+       ( (*pd2)->prevline == (*pd1)->prevline || (*pd2)->prevline == (*pd1)->nextline ))
+       line = (*pd2)->prevline;
+    else if ( (*pd2)->nextline != NULL && 
+       ( (*pd2)->nextline == (*pd1)->prevline || (*pd2)->nextline == (*pd1)->nextline ))
+       line = (*pd2)->nextline;
     else
 return( 0 );
 
@@ -2575,11 +2585,16 @@ static int stem_cmp( const void *_p1, const void *_p2 ) {
     struct stemdata * const *st1 = _p1, * const *st2 = _p2;
     double start1, end1, start2, end2;
     
+    /* Sort each stem along an axis determined by that stem's own
+     * orientation, so that the comparison is symmetric */
     if ( fabs( (*st1)->unit.x ) > fabs( (*st1)->unit.y )) {
        start1 = (*st1)->right.y; end1 = (*st1)->left.y;
-       start2 = (*st2)->right.y; end2 = (*st2)->left.y;
     } else {
        start1 = (*st1)->left.x; end1 = (*st1)->right.x;
+    }
+    if ( fabs( (*st2)->unit.x ) > fabs( (*st2)->unit.y )) {
+       start2 = (*st2)->right.y; end2 = (*st2)->left.y;
+    } else {
        start2 = (*st2)->left.x; end2 = (*st2)->right.x;
     }